package com.it235.handler;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.it235.pojo.AreaBean;
import com.it235.pojo.CityBean;
import com.it235.pojo.ProvinceBean;
import com.it235.pojo.StreetBean;

public class GrabHandler {

	//统计局数据地址：http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/index.html
	
	//省数据根目录
    public final static String ROOT_HTML_URL = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/";

    /**
     * 解析省份信息
     * @throws IOException
     */
    public static List<ProvinceBean> analysisProvince() throws IOException {
    	Document rootDoc = Jsoup.connect(ROOT_HTML_URL + "index.html").maxBodySize(0).get();
        //直接获取省市区tr，该tr总共有4个
        Elements provincetr = rootDoc.getElementsByClass("provincetr");
        List<ProvinceBean> provinces = new ArrayList<ProvinceBean>();
        for (Element tr : provincetr) {
        	//获取td，一个城市一个td
        	Elements tds = tr.select("td");
        	for (Element td : tds) {
				//重要的信息为链接和链接内容
        		Element a = td.select("a").get(0);
        		//获取下面的链接及链接文本
        		String href = a.attr("href");
        		String text = a.text();
        		
        		//保存省份信息
        		ProvinceBean provinceBean = new ProvinceBean();
        		//短编码
        		String shortPCode = href.replace(".html", "");
        		//6位长编码
        		String provinceCode = href.replace(".html", "") + "0000";
        		provinceBean.setProvinceCode(provinceCode);
        		provinceBean.setProvinceName(text);
        		
        		//TODO 解析当前省份下的城市数据，并存储到当前省份中
				List<CityBean> analysisCity = analysisCity(href , provinceCode , shortPCode );
				provinceBean.setCitys(analysisCity);
				provinces.add(provinceBean);
			}
		}
        return provinces;
    }
    
    /**
     * 
     * @param grabHref 查询城市的链接
     * @param shortPCode 省短Code，用于接到链接的拼接
     * @param provinceCode 省代码
     * @throws IOException
     */
    public static List<CityBean> analysisCity(String grabHref, String provinceCode , String shortPCode) throws IOException {
    	//点击进入市信息层
		Document cityDoc = Jsoup.connect(ROOT_HTML_URL + grabHref).maxBodySize(0).get();
		Elements citytr = cityDoc.getElementsByClass("citytr");
		List<CityBean> citys = new ArrayList<CityBean>();
        for (Element tr : citytr) {
        	Elements tds = tr.select("td");
        	//暂存第丢一个td中的内容
        	String tempCode = "";
        	for (int i = 0; i < tds.size(); i++) {
        		Element td = tds.get(i);
        		//重要的信息为链接和链接内容
        		Element a = td.select("a").get(0);
        		//一个tr中有2个td，第一个td为编码，第二个td为名称
    			String text = a.text();
        		if(i == 1) {
        			//获取下面的链接及链接文本
            		String href = a.attr("href");
        			CityBean cityBean = new CityBean();
            		cityBean.setProvinceCode(provinceCode);
            		cityBean.setCityCode(tempCode);
            		cityBean.setCityName(text);
            		
            		//解析当前城市下的区县数据，并存储到当前城市对象中
					List<AreaBean> analysisArea = analysisArea(href , tempCode , shortPCode);
					//TODO 这里我直接存在内存中
					cityBean.setAreas(analysisArea);
					citys.add(cityBean);
        		} else {
        			//第一个td，只取出编码暂存即可
        			//去除末尾的6个0
        			tempCode = text.substring(0 , text.length() - 6);
        		}
			}
        }
        return citys;
    }
    
    /**
     * 根据城市解析区县数据
     * @param grabHref 城市下的区县数据链接
     * @param cityCode 城市代码
     * @param shortPCode 省短代码，用于解析街道时URL拼接
     * @return 当前城市下的所有区县数据
     * @throws IOException
     */
    public static List<AreaBean> analysisArea(String grabHref , String cityCode ,String shortPCode) throws IOException {
    	//点击进入区县信息层，并设置超时时间
		Document areaDoc = Jsoup.connect(ROOT_HTML_URL + grabHref).timeout(70000).maxBodySize(0).get();
		Elements countytr = areaDoc.getElementsByClass("countytr");
		List<AreaBean> areas = new ArrayList<AreaBean>();
        for (Element tr : countytr) {
        	Elements tds = tr.select("td");
        	//暂存第丢一个td中的内容
        	String tempCode = "";
        	for (int i = 0; i < tds.size(); i++) {
        		Element td = tds.get(i);
        		//重要的信息为链接和链接内容
        		Elements select = td.select("a");
        		if(select == null || select.size() == 0)break;//市辖区无链接且不存储
        		Element a = select.get(0);
        		//一个tr中有2个td，第一个td为编码，第二个td为名称
    			String text = a.text();
        		if(i == 1) {
        			//获取下面的链接及链接文本
            		String href = a.attr("href");
        			AreaBean areaBean = new AreaBean();
        			areaBean.setAreaCode(tempCode);
        			areaBean.setCityCode(cityCode);
        			areaBean.setAreaName(text);
        			
        			//根据区县解析街道数据
            		List<StreetBean> analysisStreet = analysisStreet(href , tempCode , shortPCode );
            		//TODO 这里我直接存在内存中
            		areaBean.setStreets(analysisStreet);
            		areas.add(areaBean);
        		} else {
        			//第一个td，只取出编码暂存即可
        			//去除末尾的6个0
        			tempCode = text.substring(0 , text.length() - 6);
        		}
        		
			}
        }
		System.out.println(areas);
        return areas;
    }
    
    /**
     * 爬取街道
     * @param grabHref 街道数据链接
     * @param areaCode 区县code
     * @param shortPCode 省短code，用于拼接街道数据链接
     * @throws IOException
     * @return 返回当前区县下的所有街道
     */
    public static List<StreetBean> analysisStreet(String grabHref, String areaCode ,String shortPCode) throws IOException {
    	//点击进入街道信息层，此处的url和之前不太一样，需要手动拼接
		Document areaDoc = Jsoup.connect(ROOT_HTML_URL + shortPCode + "/" + grabHref).timeout(1000000).maxBodySize(0).get();
		Elements towntr = areaDoc.getElementsByClass("towntr");
		List<StreetBean> streets = new ArrayList<StreetBean>();
        for (Element tr : towntr) {
        	Elements tds = tr.select("td");
        	//暂存第丢一个td中的内容
        	String tempCode = "";
        	for (int i = 0; i < tds.size(); i++) {
        		Element td = tds.get(i);
        		//重要的信息为链接和链接内容
        		Element a = td.select("a").get(0);
        		//一个tr中有2个td，第一个td为编码，第二个td为名称
    			String text = a.text();
        		if(i == 1) {
        			//获取链接及链接文本
            		String href = a.attr("href");
            		StreetBean streetBean = new StreetBean();
            		streetBean.setAreaCode(areaCode);
            		streetBean.setStreetCode(tempCode);
            		streetBean.setStreetName(text);
            		streets.add(streetBean);
            		//下面还有居委会一级，一般作用不大，这里不做爬取
            		//analysisJWH(href , tempCode);
        		} else {
        			//第一个td，只取出编码暂存即可
        			tempCode = text;
        		}
			}
        }
		System.out.println(streets);
        return streets;
    }
    
    
	public static void main(String[] args) {
    	try {
    		long start = System.currentTimeMillis();
    		List<ProvinceBean> provinces = analysisProvince();
    		long middle = System.currentTimeMillis();
    		System.out.println("爬取总耗时：" + (middle - start)/1000 + "秒");
            	
    		//根据数据开始生成SQL语句，并写入指定的sql文件中
    		String saveDisk = "D://";
    		String provinceSQLFile = "province.sql";
    		String citySQLFile = "city.sql";
    		String areaSQLFile = "area.sql";
    		String streetSQLFile = "street.sql";
            
    		StringBuffer pSQLBuf = new StringBuffer("insert into BS_PROVINCE values ");
    		StringBuffer cSQLBuf = new StringBuffer("insert into BS_CITY values ");
    		StringBuffer aSQLBuf = new StringBuffer("insert into BS_AREA values ");
    		StringBuffer sSQLBuf = new StringBuffer("insert into BS_STREET values ");
    		
    		//存储省份SQL文件
    		for (int i = 0; i < provinces.size(); i++) {
    			ProvinceBean tempp = provinces.get(i);
    			pSQLBuf.append("(" + (i + 1) + " , '" + tempp.getProvinceCode() + "' , '" + tempp.getProvinceName() + "' , 1),");
    			
    			List<CityBean> citys = tempp.getCitys();
    			for (int j = 0; j < citys.size(); j++) {
    				CityBean tempc = citys.get(i);
    				cSQLBuf.append("(" + (i + 1) + " , '" + tempc.getCityCode() + "' , '" + tempc.getCityName() + "' , '" + tempc.getProvinceCode() + "' , 1),");
    				
    				List<AreaBean> areas = tempc.getAreas();
    				for (int k = 0; k < areas.size(); k++) {
    					AreaBean tempa = areas.get(k);
    					aSQLBuf.append("(" + (i + 1) + " , '" + tempa.getAreaCode() + "' , '" + tempa.getAreaName() + "' , '" + tempa.getCityCode() + "' , 1),");
        				
    					List<StreetBean> streets = tempa.getStreets();
        				for (int l = 0; l < streets.size(); l++) {
        					StreetBean temps = streets.get(l);
        					sSQLBuf.append("(" + (i + 1) + " , '" + temps.getStreetCode() + "' , '" + temps.getStreetName() + "' , '" + tempa.getAreaCode() + "' , 1),");
						}
					}
				}
			}
    		
    		//存储到磁盘
    		FileOutputStream pfos = new FileOutputStream(saveDisk + provinceSQLFile);
    		BufferedOutputStream pbos = new BufferedOutputStream(pfos);
    		pbos.write(pSQLBuf.substring(0 , pSQLBuf.length() - 1).getBytes());
    		pbos.close();
    		pfos.close();
    		

    		FileOutputStream cfos = new FileOutputStream(saveDisk + citySQLFile);
    		BufferedOutputStream cbos = new BufferedOutputStream(cfos);
    		cbos.write(cSQLBuf.substring(0 , cSQLBuf.length() - 1).getBytes());
    		cbos.close();
    		cfos.close();
    		
    		
    		FileOutputStream afos = new FileOutputStream(saveDisk + areaSQLFile);
    		BufferedOutputStream abos = new BufferedOutputStream(afos);
    		abos.write(aSQLBuf.substring(0 , aSQLBuf.length() - 1).getBytes());
    		abos.close();
    		afos.close();
    		
    		FileOutputStream sfos = new FileOutputStream(saveDisk + streetSQLFile);
    		BufferedOutputStream sbos = new BufferedOutputStream(sfos);
    		sbos.write(sSQLBuf.substring(0 , sSQLBuf.length() - 1).getBytes());
    		sbos.close();
    		sfos.close();
    		
    		long end = System.currentTimeMillis();
    		System.out.println("拼接读写总耗时：" + (end - middle)/1000 + "秒");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
